import re
import json
import chardet
from utils import get_page
from pyquery import PyQuery as pq


class CrawlMetaclass(type):
    """
    元类,Crawler类中加入
    __CrawlFunc__和CrrawlFuncCount__两个参数
    分别表示爬虫函数,和爬虫函数数量
    """
    def __new__(cls, name, bases, attrs):
        count = 0
        attrs['__CrawlFunc__'] = []
        for k, v in attrs.items():
            if 'crawl_' in k:
                attrs['__CrawlFunc__'].append(k)
                count += 1
        attrs['__CrawlFuncCount__'] = count
        return type.__new__(cls, name, bases, attrs)

class Crawler(object, metaclass=CrawlMetaclass):

    def get_proxies(self, callback):
        """
        根据参数callback，函数名称字符串，返回执行函数对象
        """
        proxies = []
        for proxy in eval("self.{}()".format(callback)):
            print('成功获取代理', proxy)
            proxies.append(proxy)
        return proxies

    def crawl_daili(self, page_count=4):
        """
        获取代理66
        ：param page_count: 页码
        ：reutrn: 代理
        """
        start_url = 'http://www.66ip.cn/{}.html'
        urls = [start_url.format(page) for page in range(1, page_count + 1)]
        for url in urls:
            print('Crawling', url)
            html = get_page(url)
            if html:
                doc = pq(html)
                trs = doc('.containerbox table tr:gt(0)').items()
                for tr in trs:
                    ip = tr.find('td:nth-child(1)').text()
                    port = tr.find('td:nth-child(2)').text()
                    addrs = tr.find('td:nth-child(3)').text()
                    yield '{} {}:{}'.format(addrs, ip, port)

    def crawl_kuaidail(self, page_count=1):
        """
        获取代理kuaidail
        :return: 代理
        """
        start_url = 'https://www.kuaidaili.com/free/inha/{}'
        urls = [start_url.format(page) for page in range(1, page_count + 1)]
        for url in urls:
            print('Crawling', url)
            html = get_page(url)
            if html:
                doc = pq(html)
                trs = doc('#list table tr:gt(0)').items()
                for tr in trs:
                    ip = tr.find('td:nth-child(1)').text()
                    port = tr.find('td:nth-child(2)').text()
                    position = ''.join(tr.find('td:nth-child(5)').text().split())
                    yield '{} {}:{}'.format(position, ip, port)

    def crawl_goubanjia(self):
        """
        获取代理Goubanjia
        :return: 代理
        """
        start_url = "http://www.goubanjia.com/"
        html = get_page(start_url)
        if html:
            doc = pq(html)
            trs = doc('.span12 table tr:gt(0)').items()
            for tr in trs:
                ip_port = tr.find('td:nth-child(1)').remove('p').text().replace('\n','')
                # ip_port = ip_port.find('span,div')
                position = tr.find('td:nth-child(4)').text()
                yield '{} {}'.format(''.join(position.split()), ip_port)

    def crawl_ip181(self):
        """
        获取xdaili
        :return: 代理
        """
        start_url = 'http://www.ip181.com/'
        datas = json.loads(get_page(start_url))['RESULT']
        for data in datas:
            port = data['port']
            position = ''.join(data['position'].split())
            ip = data['ip']
            yield '{} {}:{}'.format(position, ip, port)

    def crawl_ip3366(self):
        """
        获取云代理
        :return: 代理
        """
        for page in range(1, 2):
            start_url = 'http://www.ip3366.net/free/?stype=1&page={}'.format(page)
            html = get_page(start_url)
            data = re.compile('<tr>\s*<td>(.*?)</td>\s*<td>(.*?)</td>\s*<td>.*?</td>\s*<td>.*?</td>\s*<td>(.*?)</td>')
            re_data = data.findall(html)
            for ip, port, position in re_data:
                yield '{} {}:{}'.format(position, ip, port)
             